{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 大数据处理技巧就是讲一些字段type自动降一下省内存，pc上跑需要特别注意他的内存开销问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\sorfware_install\\python_install\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2698: DtypeWarning: Columns (12,13,14,15,19,20,81,83,85,87,93,94,95,96,97,98,99,100,105,106,108,109,111,112,114,115,117,118,120,121,123,124,126,127,129,130,132,133,135,136,138,139,141,142,144,145,147,148,150,151,153,154,156,157,160) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "gl = pd.read_csv('./data/game_logs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>number_of_game</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>v_name</th>\n",
       "      <th>v_league</th>\n",
       "      <th>v_game_number</th>\n",
       "      <th>h_name</th>\n",
       "      <th>h_league</th>\n",
       "      <th>h_game_number</th>\n",
       "      <th>v_score</th>\n",
       "      <th>...</th>\n",
       "      <th>h_player_7_name</th>\n",
       "      <th>h_player_7_def_pos</th>\n",
       "      <th>h_player_8_id</th>\n",
       "      <th>h_player_8_name</th>\n",
       "      <th>h_player_8_def_pos</th>\n",
       "      <th>h_player_9_id</th>\n",
       "      <th>h_player_9_name</th>\n",
       "      <th>h_player_9_def_pos</th>\n",
       "      <th>additional_info</th>\n",
       "      <th>acquisition_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18710504</td>\n",
       "      <td>0</td>\n",
       "      <td>Thu</td>\n",
       "      <td>CL1</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>FW1</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>Ed Mincher</td>\n",
       "      <td>7.0</td>\n",
       "      <td>mcdej101</td>\n",
       "      <td>James McDermott</td>\n",
       "      <td>8.0</td>\n",
       "      <td>kellb105</td>\n",
       "      <td>Bill Kelly</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18710505</td>\n",
       "      <td>0</td>\n",
       "      <td>Fri</td>\n",
       "      <td>BS1</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>WS3</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>Asa Brainard</td>\n",
       "      <td>1.0</td>\n",
       "      <td>burrh101</td>\n",
       "      <td>Henry Burroughs</td>\n",
       "      <td>9.0</td>\n",
       "      <td>berth101</td>\n",
       "      <td>Henry Berthrong</td>\n",
       "      <td>8.0</td>\n",
       "      <td>HTBF</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18710506</td>\n",
       "      <td>0</td>\n",
       "      <td>Sat</td>\n",
       "      <td>CL1</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "      <td>RC1</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>Pony Sager</td>\n",
       "      <td>6.0</td>\n",
       "      <td>birdg101</td>\n",
       "      <td>George Bird</td>\n",
       "      <td>7.0</td>\n",
       "      <td>stirg101</td>\n",
       "      <td>Gat Stires</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18710508</td>\n",
       "      <td>0</td>\n",
       "      <td>Mon</td>\n",
       "      <td>CL1</td>\n",
       "      <td>na</td>\n",
       "      <td>3</td>\n",
       "      <td>CH1</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>Ed Duffy</td>\n",
       "      <td>6.0</td>\n",
       "      <td>pinke101</td>\n",
       "      <td>Ed Pinkham</td>\n",
       "      <td>5.0</td>\n",
       "      <td>zettg101</td>\n",
       "      <td>George Zettlein</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18710509</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue</td>\n",
       "      <td>BS1</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "      <td>TRO</td>\n",
       "      <td>na</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>Steve Bellan</td>\n",
       "      <td>5.0</td>\n",
       "      <td>pikel101</td>\n",
       "      <td>Lip Pike</td>\n",
       "      <td>3.0</td>\n",
       "      <td>cravb101</td>\n",
       "      <td>Bill Craver</td>\n",
       "      <td>6.0</td>\n",
       "      <td>HTBF</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 161 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       date  number_of_game day_of_week v_name v_league  v_game_number h_name  \\\n",
       "0  18710504               0         Thu    CL1       na              1    FW1   \n",
       "1  18710505               0         Fri    BS1       na              1    WS3   \n",
       "2  18710506               0         Sat    CL1       na              2    RC1   \n",
       "3  18710508               0         Mon    CL1       na              3    CH1   \n",
       "4  18710509               0         Tue    BS1       na              2    TRO   \n",
       "\n",
       "  h_league  h_game_number  v_score       ...         h_player_7_name  \\\n",
       "0       na              1        0       ...              Ed Mincher   \n",
       "1       na              1       20       ...            Asa Brainard   \n",
       "2       na              1       12       ...              Pony Sager   \n",
       "3       na              1       12       ...                Ed Duffy   \n",
       "4       na              1        9       ...            Steve Bellan   \n",
       "\n",
       "   h_player_7_def_pos h_player_8_id  h_player_8_name h_player_8_def_pos  \\\n",
       "0                 7.0      mcdej101  James McDermott                8.0   \n",
       "1                 1.0      burrh101  Henry Burroughs                9.0   \n",
       "2                 6.0      birdg101      George Bird                7.0   \n",
       "3                 6.0      pinke101       Ed Pinkham                5.0   \n",
       "4                 5.0      pikel101         Lip Pike                3.0   \n",
       "\n",
       "  h_player_9_id  h_player_9_name  h_player_9_def_pos  additional_info  \\\n",
       "0      kellb105       Bill Kelly                 9.0              NaN   \n",
       "1      berth101  Henry Berthrong                 8.0             HTBF   \n",
       "2      stirg101       Gat Stires                 9.0              NaN   \n",
       "3      zettg101  George Zettlein                 1.0              NaN   \n",
       "4      cravb101      Bill Craver                 6.0             HTBF   \n",
       "\n",
       "  acquisition_info  \n",
       "0                Y  \n",
       "1                Y  \n",
       "2                Y  \n",
       "3                Y  \n",
       "4                Y  \n",
       "\n",
       "[5 rows x 161 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gl.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(171907, 161)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gl.shape # 数据量很大"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 171907 entries, 0 to 171906\n",
      "Columns: 161 entries, date to acquisition_info\n",
      "dtypes: float64(77), int64(6), object(78)\n",
      "memory usage: 211.2+ MB\n"
     ]
    }
   ],
   "source": [
    "# 查看数据及多大\n",
    "gl.info()#这个执行结果只能初步估计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 171907 entries, 0 to 171906\n",
      "Columns: 161 entries, date to acquisition_info\n",
      "dtypes: float64(77), int64(6), object(78)\n",
      "memory usage: 860.5 MB\n"
     ]
    }
   ],
   "source": [
    "gl.info(memory_usage='deep') #需要看具体详细数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 查看gl中每一种数据占用内存的情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "float64 1.2947326073279748\n",
      "int64 1.1241934640066964\n",
      "object 9.514454069016855\n"
     ]
    }
   ],
   "source": [
    "for dtype in ['float64','int64','object']:\n",
    "    select_type=gl.select_dtypes(include=[dtype])#注意df中每个当前值基础数据的列集合\n",
    "    memory_usage_b=select_type.memory_usage(deep=True).mean() #如何在df中取每种类型数据内存情况\n",
    "    memory_usage_mb=memory_usage_b/(1024**2)\n",
    "    print(dtype,memory_usage_mb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##### 以上数据说明 float64【77】 object【78】占用大头，需要向下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 举例子查看每种类型取值范围"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Machine parameters for uint8\n",
      "---------------------------------------------------------------\n",
      "min = 0\n",
      "max = 255\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Machine parameters for int8\n",
      "---------------------------------------------------------------\n",
      "min = -128\n",
      "max = 127\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Machine parameters for int16\n",
      "---------------------------------------------------------------\n",
      "min = -32768\n",
      "max = 32767\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Machine parameters for int32\n",
      "---------------------------------------------------------------\n",
      "min = -2147483648\n",
      "max = 2147483647\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Machine parameters for int64\n",
      "---------------------------------------------------------------\n",
      "min = -9223372036854775808\n",
      "max = 9223372036854775807\n",
      "---------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "int_types = ['uint8','int8','int16','int32','int64']\n",
    "for it in int_types :\n",
    "     print(np.iinfo(it)) # 在np中打印每种数字取值范围"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#从上面可以看到其实一般我们只要将64降到32就可以了，原因一般数值完全满足的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 封装df的三种类型的占用内存情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mem_usage(pandas_obj):\n",
    "     if isinstance(pandas_obj,pd.DataFrame):\n",
    "        usage_b = pandas_obj.memory_usage(deep=True).sum()\n",
    "     else:\n",
    "        usage_b = pandas_obj.memory_usage(deep=True)\n",
    "     usage_mb=usage_b/1024**2\n",
    "     return '{:03.2f}MB'.format(usage_mb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.87MB\n",
      "1.48MB\n"
     ]
    }
   ],
   "source": [
    "#降int类型\n",
    "gl_int = gl.select_dtypes(include = ['int64'])\n",
    "coverted_int = gl_int.apply(pd.to_numeric,downcast='unsigned')\n",
    "print (mem_usage(gl_int))\n",
    "print (mem_usage(coverted_int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.99MB\n",
      "50.49MB\n"
     ]
    }
   ],
   "source": [
    "#降float类型\n",
    "gl_float = gl.select_dtypes(include = ['float64'])\n",
    "coverted_float = gl_float.apply(pd.to_numeric,downcast='float')\n",
    "print (mem_usage(gl_float))\n",
    "print (mem_usage(coverted_float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#修改正df 对应数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "copy_gl=gl.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "860.50MB\n",
      "803.61MB\n"
     ]
    }
   ],
   "source": [
    "\n",
    "copy_gl[coverted_int.columns] = coverted_int\n",
    "copy_gl[coverted_float.columns] = coverted_float\n",
    "print(mem_usage(gl))\n",
    "print(mem_usage(copy_gl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# object 整改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>v_name</th>\n",
       "      <th>v_league</th>\n",
       "      <th>h_name</th>\n",
       "      <th>h_league</th>\n",
       "      <th>day_night</th>\n",
       "      <th>completion</th>\n",
       "      <th>forefeit</th>\n",
       "      <th>protest</th>\n",
       "      <th>park_id</th>\n",
       "      <th>...</th>\n",
       "      <th>h_player_6_id</th>\n",
       "      <th>h_player_6_name</th>\n",
       "      <th>h_player_7_id</th>\n",
       "      <th>h_player_7_name</th>\n",
       "      <th>h_player_8_id</th>\n",
       "      <th>h_player_8_name</th>\n",
       "      <th>h_player_9_id</th>\n",
       "      <th>h_player_9_name</th>\n",
       "      <th>additional_info</th>\n",
       "      <th>acquisition_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>171907</td>\n",
       "      <td>171907</td>\n",
       "      <td>171907</td>\n",
       "      <td>171907</td>\n",
       "      <td>171907</td>\n",
       "      <td>140150</td>\n",
       "      <td>116</td>\n",
       "      <td>145</td>\n",
       "      <td>180</td>\n",
       "      <td>171907</td>\n",
       "      <td>...</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>140838</td>\n",
       "      <td>1456</td>\n",
       "      <td>140841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>7</td>\n",
       "      <td>148</td>\n",
       "      <td>7</td>\n",
       "      <td>148</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>116</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>245</td>\n",
       "      <td>...</td>\n",
       "      <td>4774</td>\n",
       "      <td>4720</td>\n",
       "      <td>5253</td>\n",
       "      <td>5197</td>\n",
       "      <td>4760</td>\n",
       "      <td>4710</td>\n",
       "      <td>5193</td>\n",
       "      <td>5142</td>\n",
       "      <td>332</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Sat</td>\n",
       "      <td>CHN</td>\n",
       "      <td>NL</td>\n",
       "      <td>CHN</td>\n",
       "      <td>NL</td>\n",
       "      <td>D</td>\n",
       "      <td>19560810,,7,6,48</td>\n",
       "      <td>H</td>\n",
       "      <td>V</td>\n",
       "      <td>STL07</td>\n",
       "      <td>...</td>\n",
       "      <td>grimc101</td>\n",
       "      <td>Charlie Grimm</td>\n",
       "      <td>grimc101</td>\n",
       "      <td>Charlie Grimm</td>\n",
       "      <td>lopea102</td>\n",
       "      <td>Al Lopez</td>\n",
       "      <td>spahw101</td>\n",
       "      <td>Warren Spahn</td>\n",
       "      <td>HTBF</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>28891</td>\n",
       "      <td>8870</td>\n",
       "      <td>88866</td>\n",
       "      <td>9024</td>\n",
       "      <td>88867</td>\n",
       "      <td>82724</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>90</td>\n",
       "      <td>7022</td>\n",
       "      <td>...</td>\n",
       "      <td>427</td>\n",
       "      <td>427</td>\n",
       "      <td>491</td>\n",
       "      <td>491</td>\n",
       "      <td>676</td>\n",
       "      <td>676</td>\n",
       "      <td>339</td>\n",
       "      <td>339</td>\n",
       "      <td>1112</td>\n",
       "      <td>140841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       day_of_week  v_name v_league  h_name h_league day_night  \\\n",
       "count       171907  171907   171907  171907   171907    140150   \n",
       "unique           7     148        7     148        7         2   \n",
       "top            Sat     CHN       NL     CHN       NL         D   \n",
       "freq         28891    8870    88866    9024    88867     82724   \n",
       "\n",
       "              completion forefeit protest park_id       ...         \\\n",
       "count                116      145     180  171907       ...          \n",
       "unique               116        3       5     245       ...          \n",
       "top     19560810,,7,6,48        H       V   STL07       ...          \n",
       "freq                   1       69      90    7022       ...          \n",
       "\n",
       "       h_player_6_id h_player_6_name h_player_7_id h_player_7_name  \\\n",
       "count         140838          140838        140838          140838   \n",
       "unique          4774            4720          5253            5197   \n",
       "top         grimc101   Charlie Grimm      grimc101   Charlie Grimm   \n",
       "freq             427             427           491             491   \n",
       "\n",
       "       h_player_8_id h_player_8_name h_player_9_id h_player_9_name  \\\n",
       "count         140838          140838        140838          140838   \n",
       "unique          4760            4710          5193            5142   \n",
       "top         lopea102        Al Lopez      spahw101    Warren Spahn   \n",
       "freq             676             676           339             339   \n",
       "\n",
       "       additional_info acquisition_info  \n",
       "count             1456           140841  \n",
       "unique             332                1  \n",
       "top               HTBF                Y  \n",
       "freq              1112           140841  \n",
       "\n",
       "[4 rows x 78 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gl_obj = gl.select_dtypes(include = ['object']).copy()\n",
    "gl_obj.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### obj 数值相等系列高享内存空间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Thu\n",
       "1    Fri\n",
       "2    Sat\n",
       "3    Mon\n",
       "4    Tue\n",
       "Name: day_of_week, dtype: object"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dow = gl_obj.day_of_week\n",
    "dow.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  category 是object 合并内存空间一种类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Thu\n",
       "1    Fri\n",
       "2    Sat\n",
       "3    Mon\n",
       "4    Tue\n",
       "Name: day_of_week, dtype: category\n",
       "Categories (7, object): [Fri, Mon, Sat, Sun, Thu, Tue, Wed]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dow_cat = dow.astype('category')\n",
    "dow_cat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4\n",
       "1    0\n",
       "2    2\n",
       "3    1\n",
       "4    5\n",
       "5    4\n",
       "6    2\n",
       "7    2\n",
       "8    1\n",
       "9    5\n",
       "dtype: int8"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dow_cat.head(10).cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.84MB\n",
      "0.16MB\n"
     ]
    }
   ],
   "source": [
    "print (mem_usage(dow))\n",
    "print (mem_usage(dow_cat))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 以上效果很明显，封装obj转category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "converted_obj = pd.DataFrame()\n",
    "\n",
    "for col in gl_obj.columns:\n",
    "    num_unique_values = len(gl_obj[col].unique())\n",
    "    num_total_values = len(gl_obj[col])\n",
    "    if num_unique_values / num_total_values < 0.5:\n",
    "        converted_obj.loc[:,col] = gl_obj[col].astype('category')\n",
    "    else:\n",
    "        converted_obj.loc[:,col] = gl_obj[col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "751.64MB\n",
      "51.67MB\n"
     ]
    }
   ],
   "source": [
    "print(mem_usage(gl_obj))\n",
    "print(mem_usage(converted_obj))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# date进行压缩"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'optimized_gl' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-71-db3d8a21da3f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moptimized_gl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdate\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'optimized_gl' is not defined"
     ]
    }
   ],
   "source": [
    "date = copy.date\n",
    "date[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
